import scrapy
from scrapy import FormRequest,Request
from scrapy.selector import Selector
import sys
import time
import urllib
from tutorial.items import SMTXItem
reload(sys)
sys.setdefaultencoding('utf8')

class SMTXSpider(scrapy.Spider):

    def __init__(self):
        self.file = codecs.open('MiaotianxiaCompanyinfohaveSpidered.txt','wb',encoding="utf-8")

    name = "miaotianxia_2"
    custom_settings = {'ITEM_PIPELINES':{'tutorial.pipelines.SMTX_2_Pipeline':400,}}
    allowed_domains = ["miaotianxia.com"]
    start_urls = ['https://user.miaotianxia.com/index']
    loginokurl = 'https://user.miaotianxia.com/index'
    login_url = 'https://user.miaotianxia.com/public/login'
    productinfolist =[]
    today=  time.strftime("%Y-%m-%d", time.localtime())

    for num in range(1,806):
        start_urls.append('https://www.miaotianxia.com/corp?p='+str(num))


    def parse(self, response):
        strdata =str(response.text.encode('utf-8'))
        sel = Selector(text=strdata)
        trs = sel.xpath('//table[@class="tablelist"]/tr')
        for tr in trs:
            address =tr.xpath('td[1]/a/@href').extract()
            if not address:
                continue
            productaddress ='https://www.miaotianxia.com'+ str(address[0])+"?address="+str(tr.xpath('td[2]/text()').extract()[0])
            yield scrapy.Request(productaddress,callback=self.GetDetailInfo)

    def GetDetailInfo(self,response):
        items = []
        item = SMTXItem()
        strdata =str( response.text.encode('utf-8'))
        addressstr =urllib.unquote(response.url)
        address  = addressstr[addressstr.find("address=")+8:]
        item['address'] = address
        sel = Selector(text=strdata)

        requireDesPS =sel.xpath('//*[@id="corp_intro"]')
        desc = requireDesPS.xpath('string(.)').extract()[0]
        link = response.url
        peoplevalue = sel.xpath('//ul[@id="corp_contact"]/li[1]/text()').extract()
        people = peoplevalue[0] if  peoplevalue else ''
        telvalues =  sel.xpath('//ul[@id="corp_contact"]/li[2]/text()').extract()
        tel = telvalues[0] if  telvalues else ''
        emailvalues =sel.xpath('//ul[@id="corp_contact"]/li[3]/text()').extract()
        email = emailvalues[0] if  emailvalues else ''
        item['link'] = addressstr[0:addressstr.find("address=")-1]
        item['desc'] = str(desc).strip()
        item['tel'] =   tel
        item['people'] = people
        item['email']=email
        items.append(item)
        return item


    def start_requests(self):
        return [scrapy.FormRequest("https://user.miaotianxia.com/public/login",
                               formdata={'username':'fakeyoulife','password':'fakeyoulife','referer':""},
                               callback=self.after_login)]

    def after_login(self,response):
        for url in self.start_urls:
            time.sleep(1)
            yield scrapy.Request(url)
